import json
import re
import pymongo
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
from urllib.parse import urlencode
from multiprocessing import Pool

#索引页的所有内容
def get_page_index(offset,keyword) :
    data = {
        'offset': offset,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': 20,
        'cur_tab': 3
    }
    url = 'http://www.toutiao.com/search_content/?'+ urlencode(data)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求索引出错')
        return  None

#对首页进行分析
def parse_page_json(html):
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            yield item.get('url')

#获取详细网页的html
def get_page_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        #print('请求索引详细出错',url)
        return  None

#对详细网页分析，获取标题 图片等url
def parse_page_detail(html,url):
    soup = BeautifulSoup(html,'html.parser')
    title = soup.select('title')[0].get_text()
    images_patter = re.compile('gallery:(.*?)]}',re.S)
    result = re.search(images_patter,html)
    if result:
        #print(result.group(1))
        res = result.group(1)+ "]}"
        data = json.loads(res)
        if data and 'sub_images' in data.keys():
            sub_images = data.get('sub_images')
            image = [item.get('url') for item in sub_images]

            return {
                'title': title,
                'url': url,
                'image': image
            }


def main(offset):
    html = get_page_index(offset*20,'街拍')
    for url in parse_page_json(html):
        html = get_page_detail(url)
        if html :
            reslut = parse_page_detail(html,url)
            if reslut :
                print(reslut)

if __name__ == '__main__':
    pool = Pool()
    pool.map(main,[offset for offset in range(10)])